package crawler.cnki.task;

import crawler.cnki.selenium.SeleniumDownloader;
import crawler.cnki.selenium.WebDriverPool;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.RedisScheduler;
import us.codecraft.webmagic.selector.Html;

import java.io.UnsupportedEncodingException;
import java.util.List;

@Component
public class ProjectProcessor implements PageProcessor {
    @Autowired
    private DataPipeline dataPipeline;
    @Autowired
    private JedisPool jedisPool;

    private String nstrHead[] = {"http://s.wanfangdata.com.cn/nstr?q=%E5%9F%BA%E4%BA%8E",
            "http://s.wanfangdata.com.cn/nstr?q=%E7%A0%94%E7%A9%B6",
            "http://s.wanfangdata.com.cn/nstr?q=%E8%AE%A1%E7%AE%97",
            "http://s.wanfangdata.com.cn/nstr?q=%E8%87%AA%E5%8A%A8",
            "http://s.wanfangdata.com.cn/nstr?q=%E6%8A%80%E6%9C%AF",
            "http://s.wanfangdata.com.cn/nstr?q=%E7%8E%AF%E5%A2%83",
            "http://s.wanfangdata.com.cn/nstr?q=%E8%83%BD%E6%BA%90"};
    String root = "http://s.wanfangdata.com.cn/nstr?q";

    @Scheduled(fixedDelay = 1000 * 1000)
    public void PatentStarter() {
        Spider.create(new ProjectProcessor())
                .setUUID("Project")
                .setScheduler(new RedisScheduler(jedisPool))
                .addUrl(nstrHead)
                .setDownloader(new SeleniumDownloader(0, PoolUtil.getWebDriverPool(), new DataAction()))
                .addPipeline(dataPipeline)
                .thread(1)
                .run();

    }

    @Override
    public void process(Page page) {
        Html html = page.getHtml();
        String currentUrl = page.getUrl().toString();
        //如果是学者门户，将保存信息到学者
        if (currentUrl.contains("trend.") && currentUrl.contains("scholarName")) {
            page.putField("itemType", 4);
            Document scholarInfo = Jsoup.parse(html.css("div.scholar-infor").toString());
            //学者姓名
            String name = scholarInfo.select("h3.scholar-name-show").text();
            page.putField("name", name);
            //学者机构
            String organization = scholarInfo.select("h3.unit-name").text();
            page.putField("organization", organization);

        }
        //如果是科研项目
        else if (currentUrl.contains("d.wanfangdata") && currentUrl.contains("nstr/")) {
            page.putField("itemType", 3);
            Document detailInfo = Jsoup.parse(html.css("div.detailInfo").toString());
            //获取标题
            String title = detailInfo.select("div.detailTitle span.detailTitleCN").text();
            page.putField("title", title);

            //作者
            String author = detailInfo.select("div.author div.itemUrl").text().replaceAll(" ", ",");
            page.putField("author", author);

            //摘要
            Elements summarys = detailInfo.select("div.summary");
            if (!summarys.isEmpty()) {
                String summary = summarys.first().ownText();
                page.putField("summary", summary);
            }

            //工作单位
            Elements companies = detailInfo.select("div.organization div.itemUrl span");
            StringBuilder company = new StringBuilder();
            for (int i = 0; i < companies.size(); i = i + 2) {
                company.append(companies.get(i).text() + ",");
            }
            if (company.length() > 0)
                page.putField("company", company.substring(0, company.length() - 1));

            //关键词
            Elements keys = detailInfo.select("div.keyword div.itemUrl");
            if (keys.size() == 0)
                keys = detailInfo.select("div.keywordEN div.itemUrl");
            String keywords = keys.text().replaceAll(" ", ";");
            page.putField("keywords", keywords);
            String[] split = keywords.split(";");
            if(split.length>=1) {
                try {
                    page.addTargetRequest(root+java.net.URLEncoder.encode(split[0],"utf-8"));
                } catch (UnsupportedEncodingException e) {
                    e.printStackTrace();
                }
            }

            //立项批准年
            int year = Integer.parseInt(detailInfo.select("div.approvalDate div.itemUrl").text());
            page.putField("year", year);

            //公开范围
            String scope = detailInfo.select("div.openRange div.itemUrl").text();
            page.putField("scope", scope);

            //计划名称
            String plan = detailInfo.select("div.planName div.itemUrl").text();
            page.putField("plan", plan);

            List<String> list = html.css("div.author div.itemUrl").links().all();
//            page.addTargetRequests(list);
            ResearcherProcessor.addurl(list);
        }
        //如果是搜索页面
        else {
            List<String> links= html.css("div.result-list-container div.title-area").links().all();
            if(links.size()==0) return;
            page.addTargetRequests(links);
            if(!currentUrl.contains("&p=")){
                page.addTargetRequest(currentUrl+"&p=2");
            }else{
                String s = currentUrl;
                int i = Integer.parseInt(s.substring(s.lastIndexOf("=")+1));
                s = s.substring(0,s.lastIndexOf("=")+1)+(i+1);
                page.addTargetRequest(s);
            }
        }
    }

    private Site site = Site.me()
            .setCharset("utf8")
            .setTimeOut(10 * 1000)
            .setRetrySleepTime(3000)
            .setRetryTimes(3);

    @Override
    public Site getSite() {
        return site;
    }
}
